#!/bin/bash
# Ubuntu 22.04.4 LTS

# for ip in 192.168.0.{121..126}; do sshpass -p 123456 scp -o StrictHostKeyChecking=no -rp ~/.ssh/ $ip:/root/; done

# controller to self
# 127.0.0.1 api.mykernel.cn
# root@192:~# scp /etc/hosts 192.168.122:/etc/hosts                                                                                                                                                                                                      100%  248   133.8KB/s   00:00
# root@192:~# scp /etc/hosts 192.168.123:/etc/hosts

# node nginx to three api server
# apt install nginx
# stream {
#         server {
#                 listen 6443;
#                 proxy_pass apiservers;
#         }
#         upstream apiservers {
#                 server 192.168.0.121:6443 max_fails=2 fail_timeout=5s;
#                 server 192.168.0.122:6443 max_fails=2 fail_timeout=5s;
#                 server 192.168.0.123:6443 max_fails=2 fail_timeout=5s;
#         }
# }

# systemctl enable nginx && systemctl restart nginx && ss -tnl
# root@192:~# scp /etc/hosts 192.168.124:/etc/hosts
# root@192:~# scp /etc/hosts 192.168.125:/etc/hosts
# root@192:~# scp /etc/hosts 192.168.126:/etc/hosts



set -ux


K8S_MAJOR=1.29
K8S_MINOR=0
K8S_RELEASE=${K8S_MAJOR}.${K8S_MINOR}
K8S_RELEASE_VERSION=$K8S_RELEASE-*
MASTER_IP=1


imageRepository=registry.aliyuncs.com/google_containers
controlPlaneEndpoint=api.mykernel.cn
podSubnet=10.244.0.0/16
serviceSubnet=10.96.0.0/12
criSocket=unix:///run/containerd/containerd.sock

# ipvs/iptables
mode=ipvs 

kubeconfig_file=init.yaml



prepare() {
  # 查看每主机的mac/product uid惟一，网络通的,防火墙没有问题
ip link
cat /sys/class/dmi/id/product_uuid
ping www.baidu.com -w 1 -c 3
nc 127.0.0.1 6443

    # How to prevent prompt that ask to restart services 
    export DEBIAN_FRONTEND=noninteractive
}

function setup_k8s_mirror() {
    # aliyun镜像源
apt-get update
# apt-transport-https 可能是一个虚拟包（dummy package）；如果是的话，你可以跳过安装这个包
apt-get install -y apt-transport-https ca-certificates curl gpg
[ -f /etc/apt/keyrings/kubernetes-apt-keyring.gpg ] || curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v$K8S_MAJOR/deb/Release.key  | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
[ -f /etc/apt/sources.list.d/kubernetes.list ] || echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v$K8S_MAJOR/deb/ /" |     tee /etc/apt/sources.list.d/kubernetes.list
}


function setup_cmds() {
    # 安装指定版本的k8s
apt update
apt-cache madison kubeadm
apt install kubeadm=$K8S_RELEASE_VERSION kubelet=$K8S_RELEASE_VERSION kubectl=$K8S_RELEASE_VERSION -y
}


function lookup_cgroup_version() {
stat -fc %T /sys/fs/cgroup/
}

function setup_containerd_mirror() {
apt-get update
# step 1: 安装必要的一些系统工具
apt-get update
apt-get -y install apt-transport-https ca-certificates curl software-properties-common
# step 2: 安装GPG证书
curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | apt-key add -
# Step 3: 写入软件源信息
add-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable" --yes 
}


function setup_containerd() {
  # 安装contaienrd
apt-cache madison containerd.io
apt install containerd.io=1.6.28-1 -y
}

function setup_kernel_modules_parameters() {
cat <<EOF | tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
modprobe overlay
modprobe br_netfilter

# 设置所需的 sysctl 参数，参数在重新启动后保持不变
cat <<EOF | tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
# 应用 sysctl 参数而不重新启动
sysctl --system

lsmod | grep br_netfilter
lsmod | grep overlay
sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward
}

function config_systemdCgroup_containerd() {
containerd config default > /etc/containerd/config.toml
# systemdCgroup
sed -i  "s@SystemdCgroup =.*@SystemdCgroup = true@g" /etc/containerd/config.toml
# pause
sed -i  -E "s@ sandbox_image = \"registry.k8s.io/pause:(3.6)\"@sandbox_image = \"registry.aliyuncs.com/google_containers/pause:\1\"@g" /etc/containerd/config.toml

systemctl restart containerd
ll /var/run/containerd/containerd.sock
}

function install_yq() {
   add-apt-repository ppa:rmescandon/yq --yes 
   apt update
   apt install yq -y
}

function config_ipvs_parameters() {
cat /etc/os-release
uname -a
grep -e ipvs -e nf_conntrack /lib/modules/$(uname -r)/modules.builtin
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
lsmod | grep -e ip_vs -e nf_conntrack
cut -f1 -d " "  /proc/modules | grep -e ip_vs -e nf_conntrack
cat <<EOF | tee -a /etc/modules-load.d/k8s.conf
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
EOF
}

function  config_kubeadmConfig() {
kubeadm config print  init-defaults > $kubeconfig_file
 
# IPVS
cat <<EOF | tee -a $kubeconfig_file
# https://kubernetes.io/zh-cn/docs/reference/config-api/kubeadm-config.v1beta3/
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
EOF

# DNS
cat <<EOF | tee -a $kubeconfig_file
---
apiVersion: kubelet.config.k8s.io/v1beta1
cgroupDriver: systemd
clusterDNS:
- ${serviceSubnet%.*}.10
kind: KubeletConfiguration
resolvConf: /run/systemd/resolve/resolv.conf
EOF
}
function ignore_swap() {
  # 生产环境不需要此步骤，直接禁用swap，跳过此步骤。从main函数中删除ignore_swap。
  cat /proc/swaps && swapoff -a && sed -i -E 's/(^[^#].*swap)/#\1/g' /etc/fstab
}



function setup_dns() {
  echo "127.0.0.1 $controlPlaneEndpoint" | tee -a /etc/hosts 
}
function init_k8s_with_config() {
  # kube-proxy with ipvs
yq eval 'select(.kind == "ClusterConfiguration") .networking.serviceSubnet = "'$serviceSubnet'"' -i $kubeconfig_file
yq eval 'select(.kind == "ClusterConfiguration") .networking.podSubnet = "'$podSubnet'"' -i $kubeconfig_file
yq eval 'select(.kind == "ClusterConfiguration") .controlPlaneEndpoint = "'$controlPlaneEndpoint'"' -i $kubeconfig_file
yq eval 'select(.kind == "ClusterConfiguration") .imageRepository = "'$imageRepository'"' -i $kubeconfig_file
yq eval 'select(.kind == "InitConfiguration") .nodeRegistration.criSocket = "'$criSocket'"' -i $kubeconfig_file
yq eval 'select(.kind == "InitConfiguration") .nodeRegistration.name = "'$MASTER_IP'"' -i $kubeconfig_file
yq eval 'select(.kind == "InitConfiguration") .localAPIEndpoint.advertiseAddress = "'$MASTER_IP'"' -i $kubeconfig_file


kubeadm init  --upload-certs  --config $kubeconfig_file 

}

function init_k8s_with_command() {
  # kube-proxy without ipvs, using iptables
kubeadm init  --upload-certs  --pod-network-cidr $podSubnet  --control-plane-endpoint $controlPlaneEndpoint \
--cri-socket $criSocket \
--apiserver-advertise-address $MASTER_IP \
--node-name $MASTER_IP \
--image-repository $imageRepository \
--service-cidr $serviceSubnet
}

function crictl_config_endpoint() {
crictl config --set runtime-endpoint=$criSocket
crictl config --set image-endpoint=$criSocket
crictl ps
crictl images
}


function install_kubeconfig() {
  mkdir -p $HOME/.kube
  cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  chown $(id -u):$(id -g) $HOME/.kube/config

}

function generate_join_command() {
  kubeadm token create --print-join-command > join-node-command.sh 
  echo "$(kubeadm token create --print-join-command) \
   --control-plane --certificate-key $(kubeadm init phase upload-certs --upload-certs --config $kubeconfig_file   | tail -n1)" > join-controller-node-command.sh 
}

main() {
  prepare
  setup_k8s_mirror
  setup_cmds
  lookup_cgroup_version
  setup_containerd_mirror
  setup_containerd
  setup_kernel_modules_parameters
  config_systemdCgroup_containerd
  config_ipvs_parameters
  ignore_swap
  install_yq
  config_kubeadmConfig

  crictl_config_endpoint
  setup_dns


  case $mode in
  ipvs) init_k8s_with_config ;;
  iptables) init_k8s_with_command ;;
  esac 

  install_kubeconfig
  generate_join_command

}

main 
